package com.crwal.app.service.impl;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;

import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.crwal.app.bean.News;
import com.crwal.app.service.WebCrwalService;
import com.crwal.app.utils.DateUtils;
import com.crwal.app.utils.JsoupUtil;
import com.crwal.app.utils.WordCounterUtils;

import cn.hutool.core.date.DateUtil;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
public class WebCrwalServiceImpl implements WebCrwalService{

	public List<News> natureworksllc(String company, String url, String maxDate) throws Exception{
		url = "https://www.natureworksllc.com/News-and-Events/Press-Releases";
		log.debug("访问网站url=="+url);
		List<News> list = new ArrayList<News>();
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Elements elements = document.select("html body div div div div div div.listing-press-releases");
		for(Element element : elements) {
			Elements eles = element.select("div.media-body");
			boolean stopFlag = false;//当抓取的发布时间小于最大时间置为true,停止抓取
			for(Element ele : eles) {
				Map<String, Integer> keyWordCount = new TreeMap<String, Integer>();
				keyWordCount.put("PLA", 0);
				String publishDate = ele.select(".listing-date").text();
				if(!StringUtils.isEmpty(publishDate)){
					publishDate = DateUtil.format(new Date(publishDate), "yyyy-MM-dd");
				}
				String title = ele.select("h4.media-heading").text();
				String abstractContent = ele.select("div.listing-abstract").text();
				String href = ele.select("p.listing-actionlink a").first().attr("abs:href");
				News news = new News();
				news.setCompany(company);
				news.setCompanyHref(url);
				news.setAbstractContent(abstractContent);
				news.setHref(href);
				news.setTitle(title);
				news.setPublishDate(publishDate);
				if(!StringUtils.isEmpty(href)) {
					this.natureworksllcWordCounter(href, keyWordCount);
				}
				
				for(String key : keyWordCount.keySet()){
					int value = keyWordCount.get(key);
					news.setKeywords(StringUtils.isEmpty(news.getKeywords()) ? key : (news.getKeywords()+","+key));
					news.setTimes(StringUtils.isEmpty(news.getTimes()) ? (value+"") : (news.getTimes()+","+value));
				}
				log.debug("----"+news.toString());
				if(StringUtils.isEmpty(maxDate)) {
					list.add(news);
				} else {
					if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//publishdate<最大时间
						stopFlag = true;
						break;
					}else {
						list.add(news);
					}
				}
				Thread.sleep(1000);
			}
			if(stopFlag) {
				break;
			}
		}
		return list;
	}
	
	private void natureworksllcWordCounter(String url, Map<String, Integer> keyWordCount) throws Exception {
		log.debug("统计关键字，访问url="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		//文章每段文本
		Elements elements = document.select("html body div div div div div div div.col-sm-12").select("p");
		for(Element element : elements) {
			//统计关键字出现的次数
			WordCounterUtils.keyWordCounter(element.text(), keyWordCount);
		}
		log.debug("关键字出现次数统计成功。"+url);
	}

	@Override
	public List<News> cjbio(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		Map<String,String> params = new HashMap<String,String>();
		params.put("bdId", "news");
		params.put("limitNum", "6");
		params.put("device", "PC");
		params.put("lang", "en");
		params.put("useSt", "1");
		params.put("activeSt", "1");
		params.put("pageNum", "1");
		params.put("startDt", "2020");
		params.put("startDtStart", "2020-01-01 00:00:00");
		params.put("startDtEnd", "2020-12-31 11:59:59");
		url = "https://www.cjbio.net/board/getList.do";
		log.debug("访问网站url=="+url);
		Map<String,String> cookies = new HashMap<String,String>();
		cookies.put("cookie", "_ga=GA1.2.457895142.1620959166; _gid=GA1.2.1865838938.1621996694; JSESSIONID=37756C20C749F57ADE0D23D21B2D6FEA; AWSALB=80vTr+bLm/5777DtXyH+1Jp4Jbz97s4v76Fk8me/wcUmY71Tzk3DUmBce/LtjrQyxUsYXTR8gMim2DLHe7q0l8us8OgOIojgEo1W7yWaReg81LD9sjogs8m+cE6g; AWSALBCORS=80vTr+bLm/5777DtXyH+1Jp4Jbz97s4v76Fk8me/wcUmY71Tzk3DUmBce/LtjrQyxUsYXTR8gMim2DLHe7q0l8us8OgOIojgEo1W7yWaReg81LD9sjogs8m+cE6g");
		String USER_AGENT = "User-Agent";
		String USER_AGENT_VALUE = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";
		    
		Connection conn = Jsoup.connect(url).method(Method.POST)
				.header(USER_AGENT, USER_AGENT_VALUE)
				.header("accept", "application/json, text/javascript, */*; q=0.01")
				.header("accept-encoding", "gzip, deflate, br")
				.header("accept-language", "zh-CN,zh;q=0.9")
				.header("cache-control", "no-cache")
				.header("origin", "https://www.cjbio.net")
				.header("pragma", "no-cache")
				.header("referer", "https://www.cjbio.net/en/news/news/list.do")
				.ignoreContentType(true)
				.ignoreHttpErrors(true)
				.timeout(8000);
        if (null != params) {
            conn.data(params);
        }
        if (null != cookies) {
            conn.cookies(cookies);
        }
        Response d = Jsoup.connect(url).data(params).cookies(cookies)
        		.header("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36")
				.header("accept", "application/json, text/javascript, */*; q=0.01")
				.header("accept-encoding", "gzip, deflate, br")
				.header("accept-language", "zh-CN,zh;q=0.9")
				.header("cache-control", "no-cache")
				.header("origin", "https://www.cjbio.net")
				.header("pragma", "no-cache")
				.header("referer", "https://www.cjbio.net/en/news/news/list.do")
				.header("content-length", "67")
				.header("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
				.header("sec-ch-ua-mobile","?0")
				.header("sec-fetch-dest","empty")
				.header("sec-fetch-mode","cors")
				.header("sec-fetch-site","same-origin")
				.header("x-csrf-token","96d350bb-2738-4373-abc0-ee01e7827b3d")
				.header("x-requested-with","XMLHttpRequest")
        		.ignoreContentType(true).ignoreHttpErrors(true).method(Method.POST).execute();
        log.debug("----"+d.body());
        Response response = conn.execute();
        if (response.statusCode() == 200) {
        	String data = response.body();
            log.debug("0----"+data);
        } else {
        	log.debug("----"+response.statusCode());
            return list;
        }

        
		/*url = "https://www.cjbio.net/en/news/news/list.do";
		log.debug("访问网站url=="+url);
		Document document = Jsoup.connect(url).get();
		Elements elements = document.select("ul.board-list");//select("div#contents div div.news-body");
		log.debug("----"+elements);*/
		/*String data = Constant.CJBIO_RESPONSE;
		JSONObject dataObj = JSON.parseObject(data);
		JSONArray array = dataObj.getJSONArray("contents");
		if(array != null && array.size() > 0) {
			for(int i=0; i < array.size(); i++) {
				JSONObject obj = (JSONObject) array.get(i);
				Map<String, Integer> keyWordCount = new TreeMap<String, Integer>();
				keyWordCount.put("PHA", 0);
				News news = new News();
				news.setCompany(company);
				news.setCompanyHref(url);
				news.setTitle(obj.getString("topTitle"));
				String publishDate = obj.getString("startDt").substring(0, 10);
				//log.debug("0----"+publishDate);
				if(!StringUtils.isEmpty(publishDate)) {
					news.setPublishDate(publishDate);
				}
				String href = "https://www.cjbio.net/en/news/news/view.do?postSeq=";
				Integer postSeq = obj.getInteger("postSeq");
				if(!StringUtils.isEmpty(postSeq)) {
					news.setHref(href+postSeq);
					if(!StringUtils.isEmpty(news.getHref())) {
						this.cjbioWordCounter(news.getHref(), keyWordCount);
					}
					for(String key : keyWordCount.keySet()){
						int value = keyWordCount.get(key);
						news.setKeywords(StringUtils.isEmpty(news.getKeywords()) ? key : (news.getKeywords()+","+key));
						news.setTimes(StringUtils.isEmpty(news.getTimes()) ? (value+"") : (news.getTimes()+","+value));
					}
				}
				log.debug("----"+news.toString());
				if(StringUtils.isEmpty(maxDate)) {
					list.add(news);
				} else {
					if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//当抓取的发布时间小于最大时间为true,停止抓取
						break;
					}else {
						list.add(news);
					}
				}
				Thread.sleep(1000);
			}
		}*/
		return list;
	}

	//TODO 页面加载没有标签，应该是动态请求获得的
	private void cjbioWordCounter(String url, Map<String, Integer> keyWordCount) throws Exception {
		log.debug("统计关键字，访问url="+url);
		Document document = Jsoup.connect(url).get();
		//文章每段文本
		Elements elements = document.select("html body div div div div div");
		System.out.println(elements);
		for(Element element : elements) {
			System.out.println(element.text()+"\n");
			//统计关键字出现的次数
			WordCounterUtils.keyWordCounter(element.text(), keyWordCount);
		}
		log.debug("关键字出现次数统计成功。"+url);
		
	}

	@Override
	public List<News> plasticker(String company, String url, String maxDate) throws Exception {
		url = "https://plasticker.de/news/index_en.php";
		log.debug("访问网站url=="+url);
		List<News> list = new ArrayList<News>();
		
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		//获取总记录数-总页数 
		Elements elements = document.select("html body div#normal2 table tbody tr td");
		int pageSize = 15;
		int pageNum = 1;
		if(null != elements && elements.size() > 0) {
			// 1 to 15 of 2.949 News
			String[] pageInfo = elements.get(0).text().split(" ");
			pageSize = Integer.valueOf(pageInfo[2].trim());
			int recordTotal = Integer.valueOf(pageInfo[4].replace(".", "").trim());
			pageNum = (recordTotal%pageSize == 0) ? (recordTotal/pageSize) : (recordTotal/pageSize+1);
			log.debug("plasticker分页信息-pageSize="+pageSize+", pageNum="+pageNum+", recordTotal="+recordTotal);
		}
		//获取列表内容
		for(int cur = 0; cur < pageNum; cur++ ){
			boolean stopFlag = false;//当抓取的发布时间小于最大时间置为true,停止抓取
			String pageUrl = url+"?seite="+(cur+1);
			log.debug("访问分页地址url="+pageUrl);
			Response pageResponse = JsoupUtil.get(pageUrl);
			Document doc = Jsoup.parse(pageResponse.body());
			Elements eles = doc.select("html body div#normal2 div div");//href=div.balken_news a.newstext
			//Elements abstractEles = doc.select("html body div#normal2 div div a.newstext div span");
			for(int i=0; i < eles.size() ; i++) {
				Element perEl = eles.get(i);
				if(perEl.outerHtml().contains("margin-bottom:20px;width:468px;")) {
					News news = new News();
					news.setCompany(company);
					news.setCompanyHref(pageUrl);
					Element el = perEl.select("div.balken_news a.newstext").first();
					news.setHref(el.attr("href").replace("..", "https://plasticker.de"));
					news.setTitle(el.text());
					Element abstractEle = perEl.select("div span").first();
					
					if(null == abstractEle) {
						news.setAbstractContent("");
						news.setPublishDate("");
					}else {
						news.setAbstractContent(abstractEle.text());
						String publishDate = news.getAbstractContent().substring(0, 10);
						Date date1 = DateUtils.formatDate(publishDate);
						news.setPublishDate(DateUtil.formatDate(date1));
					}
					
					log.debug("----"+news.toString());
					if(StringUtils.isEmpty(maxDate)) {
						list.add(news);
					} else {
						if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//publishdate<最大时间
							stopFlag = true;
							break;
						}else {
							list.add(news);
						}
					}
					
				}
			}
			Thread.sleep(1000);
			if(stopFlag) {
				break;
			}
		}
		return list;
	}

	@Override
	public List<News> basf(String company, String url, String maxDate) throws Exception {
		url = "https://www.basf.com/global/en/media/news-releases.html#%7B%7D";
		log.debug("访问网站url=="+url);
		List<News> list = new ArrayList<News>();
		
		//获取列表记录数
		url = "https://www.basf.com/global/en/media/news-releases/_jcr_content/root/responsivegrid/overview_copy_copy.counts.json";
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Element element = document.body();
		JSONObject objPage = JSON.parseObject(element.text());
		int pageNum = 1;
		if(null != objPage ) {
			//获取分页参数，计算页数
			Integer total = objPage.getInteger("totalHits");
			Integer initialSize = objPage.getInteger("initialPageSize");
			Integer moreSize = objPage.getInteger("loadMorePageSize");
			if(total<=initialSize) {
				pageNum = 1;
			}else {
				pageNum = (total-initialSize)%moreSize == 0 ? (1+(total-initialSize)/moreSize) : (2+(total-initialSize)/moreSize);
			}
		}
		String pageUrl = "https://www.basf.com/global/en/media/news-releases/_jcr_content/root/responsivegrid/overview_copy_copy.results.html?page=";
		for(int cur = 0; cur < pageNum; cur++) {
			boolean stopFlag = false;//当抓取的发布时间小于最大时间置为true,停止抓取
			String curPageUrl = pageUrl + cur;
			log.debug("访问分页数据url="+curPageUrl);
			Response pageResponse = JsoupUtil.get(curPageUrl);
			Document doc = Jsoup.parse(pageResponse.body());
			Elements elements = doc.select("html body div.overview-result");
			for(Element ele : elements) {
				Element dateEle = ele.select("div div.search-result__content div.search-result__header span.news-result__date").first();
				Element titleEle = doc.select("div div.search-result__content h3.search-result__title").first();
				Element abstractEle = doc.select("div div.search-result__content div.search-result__description p").first();
				Element hrefEle = doc.select("div div.search-result__content div.search-result__actions a").first();
				//检索关键词
				Map<String, Integer> keyWordCount = new TreeMap<String, Integer>();
				keyWordCount.put("ecovio", 0);
				keyWordCount.put("ecoflex", 0);
				News news = new News();
				news.setCompany(company);
				news.setCompanyHref(curPageUrl);
				if(dateEle!=null && !StringUtils.isEmpty(dateEle.text().trim())) {
					news.setPublishDate(DateUtil.format(new Date(dateEle.text().trim()), "yyyy-MM-dd"));
				}
				news.setTitle(titleEle.text());
				news.setHref(hrefEle.attr("abs:href"));
				news.setAbstractContent(abstractEle.text());
				if(!StringUtils.isEmpty(news.getHref())) {
					this.basfWordCounter(news.getHref(), keyWordCount);
				}
				for(String key : keyWordCount.keySet()){
					int value = keyWordCount.get(key);
					news.setKeywords(StringUtils.isEmpty(news.getKeywords()) ? key : (news.getKeywords()+","+key));
					news.setTimes(StringUtils.isEmpty(news.getTimes()) ? (value+"") : (news.getTimes()+","+value));
				}
				log.debug("----"+news.toString());
				if(StringUtils.isEmpty(maxDate)) {
					list.add(news);
				} else {
					if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//publishdate<最大时间
						stopFlag = true;
						break;
					}else {
						list.add(news);
					}
				}
				Thread.sleep(1000);
			}
			if(stopFlag) {
				break;
			}
			Thread.sleep(1000);
		}
		return list;
	}

	private void basfWordCounter(String url, Map<String, Integer> keyWordCount) throws Exception {
		log.debug("统计关键字，访问url="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		//文章每段文本
		Elements elements = document.select("html body div div main div div div div.cmp-text").select("p");
		for(Element element : elements) {
			//统计关键字出现的次数
			WordCounterUtils.keyWordCounter(element.text(), keyWordCount);
		}
		log.debug("关键字出现次数统计成功。"+url);
	}

	@Override
	public List<News> totalcorbion(String company, String url, String maxDate) throws Exception {
		int endYear = DateUtil.thisYear();
		int startYear = 2020;
		String pageUrl = "https://api.total-corbion.com/rest/news/GetNewsByFilter?query.selectedYear=YEAR&query.pageNumber=PAGENO&query.pageSize=PAGESIZE";
		List<News> list = new ArrayList<News>();
		
		url = "https://www.total-corbion.com/news/we-re-hiring-are-you-our-new-market-segment-leader/";
		log.debug("访问网站url=="+url);
		for(int i=endYear; i>=startYear; i--) {
			boolean stopFlag = false;//当抓取的发布时间小于最大时间置为true,停止抓取
			Integer pageSize = 5;
			Document document = Jsoup.connect(pageUrl.replace("YEAR", i+"").replace("PAGENO", "1").replace("PAGESIZE", pageSize+"")).ignoreContentType(true).get();
			Thread.sleep(2000);
			Element element = document.body();
			JSONObject objPage = JSON.parseObject(element.text());
			Integer total = objPage.getInteger("articleTotal");
			pageSize = objPage.getInteger("pageSize");
			Integer pageTotal = (total%pageSize == 0 ? total/pageSize : (1+total/pageSize));
			//log.debug("----pageTotal="+pageTotal+",total="+total);
			for(int cur = 1; cur <= pageTotal; cur++) {
				log.debug("访问分页数据-currentpage="+cur+",year="+i);
				Document doc = Jsoup.connect(pageUrl.replace("YEAR", i+"").replace("PAGENO", cur+"").replace("PAGESIZE", pageSize+"")).ignoreContentType(true).get();
				Element responseElement = doc.body();
				JSONObject responseObj = JSON.parseObject(responseElement.text());
				JSONArray array = responseObj.getJSONArray("articles");
				if(array!=null && array.size() > 0) {
					for(int j=0; j < array.size(); j++) {
						//初始化检索关键词
						Map<String, Integer> keyWordCount = new TreeMap<String, Integer>();
						keyWordCount.put("PLA", 0);
						JSONObject obj = (JSONObject) array.get(j);
						News news = new News();
						news.setCompany(company);
						news.setCompanyHref(pageUrl.replace("YEAR", i+"").replace("PAGENO", cur+"").replace("PAGESIZE", pageSize+""));
						news.setHref(obj.getString("newsLink"));
						news.setTitle(obj.getString("title"));
						String publishDate = obj.getString("newsDate").substring(0, 10);
						news.setPublishDate(publishDate);
						//news.setAbstractContent(obj.getString("content"));
						if(!StringUtils.isEmpty(news.getHref())) {
							this.totalcorbionWordCounter(news.getHref(), keyWordCount);
						}
						for(String key : keyWordCount.keySet()){
							int value = keyWordCount.get(key);
							news.setKeywords(StringUtils.isEmpty(news.getKeywords()) ? key : (news.getKeywords()+","+key));
							news.setTimes(StringUtils.isEmpty(news.getTimes()) ? (value+"") : (news.getTimes()+","+value));
						}
						log.debug("----"+news.toString());
						if(StringUtils.isEmpty(maxDate)) {
							list.add(news);
						} else {
							if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//publishdate<最大时间
								stopFlag = true;
								break;
							}else {
								list.add(news);
							}
						}
						Thread.sleep(500);
					}
				}
				Thread.sleep(2000);
				if(stopFlag) {
					break;
				}
			}
			if(stopFlag) {
				break;
			}
		}
		return list;
	}

	private void totalcorbionWordCounter(String url, Map<String, Integer> keyWordCount) throws Exception {
		log.debug("统计关键字，访问url="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		//文章每段文本
		Elements elements = document.select("html body div div div div div div div div div div div div").select("p");
		//System.out.println(elements.html());
		for(Element element : elements) {
			//System.out.println(element.html()+"\n");
			//统计关键字出现的次数
			WordCounterUtils.keyWordCounter(element.html(), keyWordCount);
		}
		log.debug("关键字出现次数统计成功。"+url);
		
	}

	@Override
	public List<News> bioplasticsnews(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		url = "https://bioplasticsnews.com/";
		log.debug("访问网站url=="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Elements elements = document.select("html body div div main article");
		
		for(Element element : elements) {
			News news = new News();
			news.setCompany(company);
			news.setCompanyHref(url);
			news.setHref(element.select("a").first().attr("abs:href"));
			news.setTitle(element.select("header h2").first().text());
			Element dateEle = element.select("footer div span time").first();
			news.setPublishDate(DateUtil.format(new Date(dateEle.text().trim()), "yyyy-MM-dd"));
			log.debug("----"+news);
			if(StringUtils.isEmpty(maxDate)) {
				list.add(news);
			} else {
				if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//当抓取的发布时间小于最大时间为true,停止抓取
					break;
				}else {
					list.add(news);
				}
			}
		}
		return list;
	}

	@Override
	public List<News> bioplasticsmagazine(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		url = "https://www.bioplasticsmagazine.com/en/";
		log.debug("访问网站url=="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Elements elements = document.select("html body div div div div div div div div div div div ul li div div.listEntryText");
		
		for(Element element : elements) {
			News news = new News();
			news.setCompany(company);
			news.setCompanyHref(url);
			Element dateEle = element.select("div.listEntryDate").first();
			news.setPublishDate(DateUtil.format(new Date(dateEle.text().trim()), "yyyy-MM-dd"));
			news.setTitle(element.select("h3.listEntryTitle").first().text());
			news.setHref(element.select("div.listEntryLink a").first().attr("abs:href"));
			news.setAbstractContent(element.select("div.listEntryDescription").first().text());
			log.debug("----"+news);
			if(StringUtils.isEmpty(maxDate)) {
				list.add(news);
			} else {
				if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//当抓取的发布时间小于最大时间为true,停止抓取
					break;
				}else {
					list.add(news);
				}
			}
		}
		return list;
	}

	//TODO 检查是否动态请求分页列表数据
	@Override
	public List<News> plasticsnews(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		url = "https://www.plasticsnews.com/news";
		String nextPageUrl = url;
		if(!StringUtils.isEmpty(nextPageUrl)) {
			plasticsnewsNextPage(company, maxDate, nextPageUrl, list);
		}
		return list;
	}
	
	private void plasticsnewsNextPage(String company, String maxDate, String url, List<News> list) throws Exception {
		log.debug("访问网站url=="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Elements elements = document.select("div.block-region-content-left div div div div.view-content div div.views-row");
		//log.debug("----"+elements);
		boolean stopFlag = false;
		for(Element element : elements) {
			News news = new News();
			news.setCompany(company);
			news.setCompanyHref(url);
			Element ele = element.select("div div div.feature-article-headline").first();
			news.setTitle(ele.text());
			news.setHref(ele.select("a").first().attr("abs:href"));
			if(!StringUtils.isEmpty(news.getHref())) {
				Thread.sleep(1000);
				String publishDate = getPlasticsnewsDate(news.getHref());
				news.setPublishDate(DateUtil.format(new Date(publishDate), "yyyy-MM-dd"));
			}
			log.debug("----"+news);
			if(StringUtils.isEmpty(maxDate)) {
				list.add(news);
			} else {
				if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//当抓取的发布时间小于最大时间为true,停止抓取
					stopFlag = true;
					break;
				}else {
					list.add(news);
				}
			}
		}
		if(stopFlag) {
			return ;
		}
		Elements pageElements = document.select("div div div ul.pager");
		String nextPageUrl = pageElements.select("li a").first().attr("abs:href");
		if(!StringUtils.isEmpty(nextPageUrl)) {
			Thread.sleep(1000);
			plasticsnewsNextPage(company, maxDate, nextPageUrl, list);
		}
	}
	
	private String getPlasticsnewsDate(String url) throws Exception {
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Element element = document.select("main div div article div div div div div.block-region-content-left div div span").first();
		//log.debug("----"+element);
		String publishDate = element.text().trim();
		return publishDate;
	}
	

	@Override
	public List<News> biobased(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		url = "https://news.bio-based.eu/";
		log.debug("访问网站url=="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Elements pageElements = document.select("div#content div#nav-below div ol li a.page");
		//获取分页信息-取最大页数
		int pageNum = Integer.valueOf(pageElements.last().attr("title"));
		String companyUrl = "https://news.bio-based.eu/page/";
		for(int i = 1; i <= pageNum; i++) {
			boolean stopFlag = this.biobasedList(company, companyUrl+i+"/", list, maxDate);
			Thread.sleep(2000);
			if(stopFlag) {
				break;
			}
		}
		return list;
	}

	private boolean biobasedList(String company, String url, List<News> list, String maxDate) throws Exception {
		log.debug("访问分页地址url=="+url);
		Response response = JsoupUtil.get(url);
		Document document = Jsoup.parse(response.body());
		Elements elements = document.select("div#content div.category-news");
		String publishDate = "";
		String temDate = "";
		boolean stopFlag = false;//当抓取的发布时间小于最大时间置为true,停止抓取
		for(Element element : elements) {
			News news = new News();
			news.setCompany(company);
			news.setCompanyHref(url);
			Element dateEle = element.select("h5").first();
			if(null != dateEle) {
				DateFormat dateFormatter = new SimpleDateFormat("dd MMM yyyy", Locale.GERMANY);
			    Date date = dateFormatter.parse(dateEle.text().trim());
			    publishDate = DateUtil.format(date, "yyyy-MM-dd");
			    temDate = publishDate;
			} else {
				publishDate = temDate;
			}
			news.setPublishDate(publishDate);
			Element titleEle = element.select("h2 a").first();
			news.setTitle(titleEle.text());
			news.setHref(titleEle.attr("abs:href"));
			Element despEle = element.select("h3 a").first();
			news.setAbstractContent(despEle.text());
			log.debug("----"+news);
			if(StringUtils.isEmpty(maxDate)) {
				list.add(news);
			} else {
				if(maxDate.compareTo(news.getPublishDate()) > 0 || maxDate.equals(news.getPublishDate())) {//publishdate<最大时间
					stopFlag = true;
					break;
				}else {
					list.add(news);
				}
			}
		}
		return stopFlag;
	}

	@Override
	public List<News> corbion(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		return list;
	}

	@Override
	public List<News> pttmcc(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		return list;
	}

	@Override
	public List<News> arkema(String company, String url, String maxDate) throws Exception {
		List<News> list = new ArrayList<News>();
		return list;
	}
	
	
}
